package com.sherler.crawler.hexun;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.sherler.crawler.hexun.jpa.HexunNews;
import com.sherler.crawler.hexun.jpa.HexunNewsReository;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by GeorgeSherler on 2017/5/18.
 */
@Component
public class HexunBasicPageProcessor implements PageProcessor {
    private Site site= Site.me().setRetryTimes(5).setSleepTime(1000).setCharset("gb2312").setTimeOut(20000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");

    private HashSet<String> urls=new HashSet<>();

    private Integer currentPageNum=1527;

    private int period=1;

    private boolean shouldGoOn=true;

    private String basicUrl;

    private String category;

    @Autowired
    private HexunNewsReository hexunNewsReository;

    /**
     *
     * @param period 周期（按天计）
     * @param url 除去去页码的其他部分
     * @param category 分类名称
     */
    public void start(int period,String url,String category){
        this.period=period;
        this.basicUrl=url;
        this.category=category;
        List<String[]> listProxyies=new ArrayList<>();
//        listProxyies.add(new String[]{"HP0B0J426G6PX5UP","348A76AA52FB9D0D","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"HY56AQ0AZ77USTFP","93C6643AF55E4C8C","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H67FTUJJF5728S1P","31326E932EF1B748","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H4D39KK56GD5D16P","DE81E39203B2EC85","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H93XDR0U501E561P","8638650F275D5E21","139.129.199.35","9010"});
//        this.site.setHttpProxyPool(listProxyies,false);
        if(this.urls!=null && basicUrl!=null && category !=null)
        {
            this.category=category;
            urls.add(basicUrl+"&cp="+currentPageNum++);
            while (urls.size()>0){
                try {
                    String firsturl=urls.iterator().next();
                    urls.remove(firsturl);
                    Spider.create(this).addUrl(firsturl).thread(5).run();
                }catch (Exception e){
                    System.out.println(e);
                }
                if(shouldGoOn){
                    urls.add(basicUrl+"&cp="+currentPageNum++);
                }
            }
        }
    }

    @Override
    public void process(Page page) {
        if(isListPage(page))
        {
            String url=page.getRequest().getUrl();
            String[] pageUrlSegments=url.split("\\?|\\&");
            System.out.println("extractURLList: "+page.getUrl());
            extractURLList(page);
        }else{
            System.out.println("extractDocument:>>>>"+page.getUrl());
            extractContent(page);
        }
        if(this.urls!=null&&urls.size()>0)
        {
            String url=urls.iterator().next();
            page.addTargetRequest(url);
            synchronized (HexunBasicPageProcessor.class){
                this.urls.remove(url);
            }
        }else if(shouldGoOn){
            addListUrl(page);
        }
    }

    private void addListUrl(Page page){

        if(basicUrl==null)
        {
            return;
        }
        String listUrl=basicUrl+"&cp="+(++currentPageNum);
        System.out.println("new list url added >>>>>"+listUrl);
        page.addTargetRequest(listUrl);
    }


    /**
     * 提取列表页的URL
     * @param page
     */
    private void extractURLList(Page page){
       String rawText=page.getRawText();
       JSONObject jsonPage=null;
       if(rawText!=null)
       {
           try{
               jsonPage=JSONObject.parseObject(rawText);
           }catch (Exception e){
               System.out.println(e);
           }
       }
       if(jsonPage==null)
       {
           System.out.println("error ::: no content");
           return;
       }
       JSONArray jsonArray=(JSONArray) jsonPage.get("result");
       for(Object news:jsonArray)
       {
           String url=((JSONObject)news).get("entityurl").toString();
           if(expired(url)){
               shouldGoOn=false;
               return;
           }else if(!crawledBefore(extractId(url))){
               synchronized (HexunBasicPageProcessor.class){
                   urls.add(url);
               }
           }
       }
    }

    /**
     * 根据url判断是否到期
     * @param url
     * @return
     */
    private boolean expired(String url){
        if(url==null)
        {
            return true;
        }
        String[] urlSegments=url.split("/");
        if(urlSegments.length<2)
        {
            return true;
        }
        String dateStr=urlSegments[urlSegments.length-2];
        if(dateStr!=null)
        {
            SimpleDateFormat sdf =   new SimpleDateFormat( "yyyy-MM-dd" );
            try{
                Date date =sdf.parse(dateStr);
                if(datesBetween(date,new Date())>period){
                    return true;
                }else{
                    return false;
                }
            }catch (Exception e){
                System.out.println(e);
            }
        }
        return true;
    }

    /**
     * 提取页面内容
     * @param page
     */
    private void extractContent(Page page){
        HexunNews hexunNews=new HexunNews();

        String title=page.getHtml().xpath("//div[@class='layout mg articleName']/h1/allText()").toString();
        hexunNews.setTitle(title);

        String dateStr=page.getHtml().xpath("//div[@class='layout mg articleName']/div/div[@class='tip fl']/span/allText()").toString();
        hexunNews.setNewsDate(dateStr);

        List<String> contents=page.getHtml().xpath("//div[@class='art_context']/div[@class='art_contextBox']/p/allText()").all();
        String content=null;
        if(contents!=null&&contents.size()>0)
        {
            content= StringUtils.join(contents,"     ");
        }
        if(content!=null&&content.trim().length()>0)
        {
            hexunNews.setContent(content);
        }
        hexunNews.setCategory(category);

        String id=extractId(page.getRequest().getUrl());
        hexunNews.setUrl(page.getRequest().getUrl());
        if(id!=null)
        {
            hexunNews.setId(id);
            hexunNewsReository.save(hexunNews);
        }else{
            System.out.println("error id null");
        }

    }

    /**
     * 判断是否抓取过
     * @param id
     * @return
     */
    private boolean crawledBefore(String id){
        if(id==null)
        {
            return true;
        }
        HexunNews hexunNews=hexunNewsReository.findOne(id);
        if(hexunNews!=null && hexunNews.getId()!=null)
        {
            return true;
        }
        return false;
    }

    /**
     * 从url中取出id
     * @param url
     * @return
     */
    private String extractId(String url){
        if(url==null)
        {
            return null;
        }
        String[] urlSegments=url.split("/");
        if(urlSegments.length<1)
        {
            return null;
        }
        return urlSegments[urlSegments.length-1].replace(".html","");
    }

    /**
     * 计算两个时间之间的日期距离
     * @param date2
     * @param date1
     * @return
     */
    private int datesBetween(Date date1, Date date2){
        Calendar calendar = Calendar.getInstance();
        long time1=0;
        long time2=0;
        try{
            calendar.setTime(date1);
            time1=calendar.getTimeInMillis();
            calendar.setTime(date2);
            time2=calendar.getTimeInMillis();
        }catch (Exception e){
            System.out.println(e);
        }
        long between_days=(time2-time1)/(1000*3600*24);

        return Integer.parseInt(String.valueOf(between_days));
    }

    /**
     * 是否为列表页
     * @param page
     * @return
     */
    private boolean isListPage(Page page){
        if(page==null)
        {
            return false;
        }
        String url = page.getRequest().getUrl();
        if(url!=null&&url.contains("newsListPageByJson")){
            return true;
        }else{
            return false;
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

}
